{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVC\n",
    "from itertools import combinations,permutations\n",
    "import time\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score\n",
    "import tables\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 特征组合 对每一列分别相乘 迪卡尔积\n",
    "def combin_func(df):\n",
    "    combinations_list = ['x', 'y', 'v', 'd',\"month\",\"day\",\"hour\",\"minute\",\"second\",\"weekday\"]\n",
    "\n",
    "    for k in range(2,len(combinations_list)+1):\n",
    "        for combination in combinations(combinations_list, k):\n",
    "            a_k,s_k,c_k,chu_k = \"\",\"\",\"\",\"\"\n",
    "            a_v,s_v,c_v,chu_v = 0,0,1,1\n",
    "            for ele in combination:\n",
    "                a_k = a_k + ele+\"_a\"\n",
    "                a_v = a_v + df[ele]\n",
    "                \n",
    "                s_k = s_k + ele+\"_s\"\n",
    "                s_v = s_v - df[ele]\n",
    "                \n",
    "                c_k = c_k + ele+\"_c\"\n",
    "                c_v = c_v * df[ele]\n",
    "                \n",
    "                chu_k = chu_k + ele+\"_chu\"\n",
    "                chu_v = chu_v / df[ele]\n",
    "            df[a_k]=a_v\n",
    "            df[s_k]=s_v\n",
    "            df[c_k]=c_v\n",
    "            df[chu_k]=chu_v\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ln\n",
    "def ln_func(df):\n",
    "    combinations_list = ['x', 'y', 'month', 'day']\n",
    "    for key in combinations_list:\n",
    "        df[\"ln_\"+key] = df[key].apply(lambda x: math.log(x))\n",
    "        df[\"log2_\"+key] = df[key].apply(lambda x: math.log(x,2))\n",
    "        df[\"log10_\"+key] = df[key].apply(lambda x: math.log(x,10))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shift\n",
    "def shift_func(df):\n",
    "    combinations_list = ['x', 'y', 'v', 'd', 'month', 'day', 'hour','minute', 'second', 'weekday']\n",
    "    for key in combinations_list:\n",
    "        df['shift_'+key] = df[key].shift(1);\n",
    "        df['shift_'+key].fillna(0, inplace = True)\n",
    "        \n",
    "        df['shift2_'+key] = df[key].shift(-1);\n",
    "        df['shift2_'+key].fillna(0, inplace = True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取重要特征的列\n",
    "feature_importance_lnshifta='data/feature_importance_lnshifta.csv'\n",
    "if os.path.exists(feature_importance_lnshifta):\n",
    "    import_columns = pd.read_csv(feature_importance_lnshifta,header=None)[0].values\n",
    "else:\n",
    "    dirpath=\"data/train/数据清洗\"\n",
    "    filelist = os.listdir(dirpath)\n",
    "    filelist = filelist[:100]\n",
    "    for index,file in enumerate(tqdm(filelist,total=len(filelist))):\n",
    "        filepath=os.path.join(dirpath,file)\n",
    "        dftmp = pd.read_csv(filepath)\n",
    "        # 加减乘除\n",
    "        dftmp=combin_func(dftmp)\n",
    "        # ln\n",
    "        dftmp = ln_func(dftmp)\n",
    "        # shift\n",
    "        dftmp = shift_func(dftmp) \n",
    "        # 删除不需要保存的列\n",
    "        dftmp.drop(['x', 'y', 'v', 'd', 'month', 'day', 'hour','minute', 'second', 'weekday',\"ship\",\"time\"],axis=1,inplace=True)\n",
    "        if index == 0:\n",
    "            df = dftmp\n",
    "        else:\n",
    "            df = df.append(dftmp,ignore_index=True)\n",
    "\n",
    "    X_train = df.drop(['type'], axis=1)\n",
    "    y_train = df[\"type\"]\n",
    "    print(y_train.value_counts(1))\n",
    "\n",
    "    model = xgb.XGBClassifier()\n",
    "    model.fit(X_train,y_train)\n",
    "\n",
    "    import_columns = X_train.columns[np.where(model.feature_importances_ != 0)]\n",
    "    print(X_train.shape,import_columns.shape)\n",
    "    #X_train[import_columns]\n",
    "\n",
    "    # 变量重要性排序可视化\n",
    "    xgb.plot_importance(model)\n",
    "    plt.show()\n",
    "\n",
    "    pd.DataFrame(import_columns).to_csv(feature_importance_lnshifta,index=False, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据目录获取文件夹中所有csv的内容\n",
    "def get_data_by_dir(dirpath,istest=False):\n",
    "    filelist = os.listdir(dirpath)\n",
    "    for index,file in enumerate(tqdm(filelist,total=len(filelist))):\n",
    "        save_file_path = os.path.join(\"data/train\",\"lnshifta\",file)\n",
    "        \n",
    "        filepath=os.path.join(dirpath,file)\n",
    "        dftmp = pd.read_csv(filepath)\n",
    "        \n",
    "        if istest:\n",
    "            save_file_path = os.path.join(\"data/test\",\"lnshifta\",file)\n",
    "\n",
    "        # 加减乘除\n",
    "        dftmp=combin_func(dftmp)\n",
    "        # ln\n",
    "        dftmp = ln_func(dftmp)\n",
    "        # shift\n",
    "        dftmp = shift_func(dftmp) \n",
    "        # 删除不需要保存的列\n",
    "        dftmp.drop(['x', 'y', 'v', 'd', 'month', 'day', 'hour','minute', 'second', 'weekday',\"ship\",\"time\",\"type\"],axis=1,inplace=True)\n",
    "        \n",
    "        dftmp = dftmp.replace([np.inf, -np.inf], 0)\n",
    "        \n",
    "        dftmp = dftmp[import_columns]\n",
    "        \n",
    "        dftmp.to_csv(save_file_path,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 76/7000 [05:56<9:25:11,  4.90s/it] ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/home/carl-hui/.virtualenvs/AI/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3319, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-7-50fad8207dda>\", line 4, in <module>\n",
      "    f1 = t.submit(get_data_by_dir,\"data/train/数据清洗\")\n",
      "  File \"/usr/lib/python3.7/concurrent/futures/_base.py\", line 623, in __exit__\n",
      "    self.shutdown(wait=True)\n",
      "  File \"/usr/lib/python3.7/concurrent/futures/thread.py\", line 216, in shutdown\n",
      "    t.join()\n",
      "  File \"/usr/lib/python3.7/threading.py\", line 1044, in join\n",
      "    self._wait_for_tstate_lock()\n",
      "  File \"/usr/lib/python3.7/threading.py\", line 1060, in _wait_for_tstate_lock\n",
      "    elif lock.acquire(block, timeout):\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/carl-hui/.virtualenvs/AI/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2034, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/carl-hui/.virtualenvs/AI/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1151, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/home/carl-hui/.virtualenvs/AI/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 319, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/home/carl-hui/.virtualenvs/AI/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 353, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/usr/lib/python3.7/inspect.py\", line 1502, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/usr/lib/python3.7/inspect.py\", line 1460, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/usr/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/usr/lib/python3.7/inspect.py\", line 739, in getmodule\n",
      "    f = getabsfile(module)\n",
      "  File \"/usr/lib/python3.7/inspect.py\", line 708, in getabsfile\n",
      "    _filename = getsourcefile(object) or getfile(object)\n",
      "  File \"/usr/lib/python3.7/inspect.py\", line 693, in getsourcefile\n",
      "    if os.path.exists(filename):\n",
      "  File \"/home/carl-hui/.virtualenvs/AI/lib/python3.7/genericpath.py\", line 19, in exists\n",
      "    os.stat(path)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=10) as t: \n",
    "    f1 = t.submit(get_data_by_dir,\"data/train/数据清洗\")\n",
    "    \n",
    "end = time.time()\n",
    "print(str(end-start))  #841.635686"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=10) as t: \n",
    "    f1 = t.submit(get_data_by_dir,\"data/test/数据清洗\",True)\n",
    "\n",
    "end = time.time()\n",
    "print(str(end-start))  #841.635686"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
